"""
Simplify Antwerp data files to a format more suitable for data.gov.be.
"""

__author__ = "Bart Hanssens <bart.hanssens@fedict.be>"

import argparse, os, logging
import json
from bs4 import BeautifulSoup

logging.basicConfig(filename='antwerpen-simplify.log', level=logging.DEBUG)

def ckanparse(infile, outfile):
	"""Parse HTML file and store it in a new (simplified) JSON file."""
	logging.info('Parsing ' + str(infile))

	data = {} 

	with open(infile, 'r') as f:
		soup = BeautifulSoup(f.read())
		
		data['title'] = soup.find('h1').get_text()

		body = soup.find('div', class_='group-left')
		p = body.find('p')
		if p:
			data['body'] = p.get_text()

		for k in ('contexts', 'urls', 'fmts', 'tags', 'license'):
			data[k] = set()	

		name = soup.find('link', rel='canonical')
		data['contexts'].add('http://opandata.antwerpen.be' + name['href'])

		node = soup.find('div', class_='node')

		div = node.find('div', class_='downloads')
		if div:
			hrefs = div.find_all('a')
			for href in hrefs:
				data['urls'].add(href['href'])
				data['fmts'].add(href['class'][0])

		for label in node.find_all('a', typeof='skos:Concept'):
			data['tags'].add(label.get_text())

	# Add defaults for required fields
	if not body in data:
		data['body'] = data['title']
	data['body'] = data['body'].strip()

	data['org'] = ['--Antwerpen']
	data['geo'] = ['----Antwerpen']
	
	data['media'] = ['Download', 'Webservice']

	data['license'].add('Zie website van de data leverancier')

	# make entries JSON serializable 
	for k in ('contexts', 'urls', 'tags', 'fmts', 'license'): 
		data[k] = list(data[k])
	
	with open(outfile, 'w') as f:
		json.dump(data, f, indent=4)


def main():	
	parser = argparse.ArgumentParser(description='Simplify Antwerp HTML files')
	parser.add_argument('--indir', help='Input directory', required=True)
	parser.add_argument('--outdir', help='Output directory', required=True)
	
	args = parser.parse_args()
	
	for f in os.listdir(args.indir):
		ckanparse(os.path.join(args.indir, f), os.path.join(args.outdir, f))
			
if __name__ == "__main__":
	main()
